# -*-coding:utf-8 -*-
from urllib.parse import quote_plus,urlparse
import botocore
from pymongo import MongoClient
from boto3.session import Session
import logging
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)



def get_database_info(database_name,collection_name):
    # 指定要获取信息的数据库
    db = client[database_name]
    # 获取数据库中所有集合
    collection = db[collection_name]
    # 从文本文件中读取 companyId
    company_ids = []
    with open('companyId.txt', 'r') as file:
        company_ids = [line.strip() for line in file]

    object_num_success = 0
    object_num_error = 0
    # 遍历所有租户挨个租户进行查询数据处理。
    for company_id in company_ids:
        company_num_success = 0
        company_num_error = 0
        # 聚合查询
        pipeline = [
            {
                "$match": {
                    "companyId":  company_id,
                    "ctime": {
                        "$gte": "2017-01-01",
                        "$lte": "2017-12-31"
                    }
                }
            },
            {
                "$project": {
                    "_id": 0,
                    "companyId": 1,
                    "httpWan": 1
                }
            }
        ]
        '''
        while true防止执行数据库超时等异常进行再次执行查询。
        '''
        while True:
            try:
                results = collection.aggregate(pipeline, allowDiskUse=True, batchSize=1000)
                for result in results:
                    httpWan = result['httpWan']

                    # 对查询的结果进行字符串处理
                    if isinstance(httpWan, bytes):
                        httpWan = httpWan.decode('utf-8')
                    parsed_url = urlparse(httpWan)
                    if isinstance(parsed_url.path, bytes):
                        path = parsed_url.path.decode('utf-8')
                    else:
                        path = parsed_url.path
                    object_path = path.lstrip('/')
                    if not object_path:
                        logging.error("The object path is empty. Skipping this entry.")
                        continue
                    print(object_path)

                    # 拷贝存储对象到另外一个桶里
                    try:
                        s3_client.copy_object(CopySource={'Bucket': bucket_name,
                                                      'Key': object_path},
                                          Bucket=bucket_des,
                                          Key=object_path)
                    except s3_client.exceptions.NoSuchKey:
                        logging.error(f"No such key: {object_path} in bucket: {bucket_name}")
                    except Exception as e:
                        logging.error(f"An error occurred: {e}")

                    # 修改存储对象为归档存储，保存在当前桶里
                    try:
                        response = s3_client.copy_object(CopySource={'Bucket': bucket_name,
                                                                     'Key': object_path},
                                                         Bucket=bucket_name,
                                                         Key=object_path,
                                                         StorageClass='GLACIER')
                        company_num_success += 1
                        object_num_success += 1
                    except botocore.exceptions.ClientError as error:
                        print(f"Error occurred: {error}")
                        company_num_error += 1
                        object_num_error += 1
                    except Exception as e:
                        print(f"error occurred: {e}")
                        company_num_error += 1
                        object_num_error += 1
                #拷贝及修改存储执行完之后从当前循环退出。
                break
            except Exception as e:
                print(f"查询过程中发生错误: {e}")
            finally:
                results.close()
        print('租户{}处理成功数据量有{},失败数据量有{}'.format(company_id, company_num_success, company_num_error))

    print('处理了成功的数据量有{},失败的数据量有{}'.format(object_num_success,object_num_error))


if __name__ == "__main__":
    #连接obs信息
    access_key = "xxxxxx"
    secret_key = "xxxxxx"
    url = "http://100.127.100.130"
    bucket_name = "ys-public"
    bucket_des = "2017"
    session = Session(access_key, secret_key)
    s3_client = session.client("s3", endpoint_url=url)

    # 指定要获取信息的数据库名称
    database_name = "rms-resource"
    collection_name = "material"
    # MongoDB 连接信息
    mongo_host = 'xxxxx'
    mongo_port = 8030
    mongo_user = 'root'
    mongo_password = 'xxxxxx'
    escaped_password = quote_plus(mongo_password)
    escaped_host = quote_plus(mongo_host)
    # 连接 MongoDB
    connection_string = f'mongodb://{mongo_user}:{escaped_password}@{mongo_host}:{mongo_port}/admin'
    client = MongoClient(connection_string)

    get_database_info(database_name, collection_name)
